{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1fe6e643-9453-4381-9445-bd471685fb96",
   "metadata": {},
   "source": [
    "## Exploring the Ethos dataset using Autolabel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80110a5b-2b3e-45e2-a2da-f6fa00200dff",
   "metadata": {},
   "source": [
    "#### Setup the API Keys for providers that you want to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "92993c83-4473-4e05-9510-f543b070c7d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# provide your own OpenAI API key here\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxxxxxxxxxxxxxxxx\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c246f85",
   "metadata": {},
   "source": [
    "#### Install the autolabel library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bc181e31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: refuel-autolabel[openai] in /opt/homebrew/lib/python3.10/site-packages (0.0.1)\n",
      "Requirement already satisfied: loguru>=0.5.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.5.3)\n",
      "Requirement already satisfied: numpy>=1.23.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.23.3)\n",
      "Requirement already satisfied: requests>=2.27.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2.28.1)\n",
      "Requirement already satisfied: datasets>=2.7.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2.11.0)\n",
      "Requirement already satisfied: langchain>=0.0.190 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.0.191)\n",
      "Requirement already satisfied: nervaluate>=0.1.8 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.1.8)\n",
      "Requirement already satisfied: pandas>=1.3.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.5.0)\n",
      "Requirement already satisfied: scikit-learn>=1.0.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.1.2)\n",
      "Requirement already satisfied: tenacity>=8.2.2 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (8.2.2)\n",
      "Requirement already satisfied: SQLAlchemy==1.4.47 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.4.47)\n",
      "Requirement already satisfied: regex>=2023.6.3 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2023.6.3)\n",
      "Requirement already satisfied: rich>=13.3.5 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (13.4.1)\n",
      "Requirement already satisfied: scipy>=1.10.1 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.10.1)\n",
      "Requirement already satisfied: pydantic>=1.10.9 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.10.9)\n",
      "Requirement already satisfied: torch>=1.10.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.13.0)\n",
      "Requirement already satisfied: matplotlib>=3.5.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (3.6.0)\n",
      "Requirement already satisfied: wget>=3.2 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (3.2)\n",
      "Requirement already satisfied: openai>=0.27.4 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.27.6)\n",
      "Requirement already satisfied: tiktoken>=0.3.3 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.3.3)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (9.0.0)\n",
      "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.3.5.1)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (4.64.1)\n",
      "Requirement already satisfied: xxhash in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (3.1.0)\n",
      "Requirement already satisfied: multiprocess in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.70.13)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (2022.10.0)\n",
      "Requirement already satisfied: aiohttp in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (3.8.3)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.14.1)\n",
      "Requirement already satisfied: packaging in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (21.3)\n",
      "Requirement already satisfied: responses<0.19 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.18.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (6.0)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (4.0.2)\n",
      "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (0.5.7)\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (2.8.4)\n",
      "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (1.2.4)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.0.5)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (4.37.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.4.4)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (9.2.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/lib/python3.10/site-packages (from pandas>=1.3.0->refuel-autolabel[openai]) (2022.2.1)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /opt/homebrew/lib/python3.10/site-packages (from pydantic>=1.10.9->refuel-autolabel[openai]) (4.3.0)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (1.26.13)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2022.9.14)\n",
      "Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in /opt/homebrew/lib/python3.10/site-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.2.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/homebrew/lib/python3.10/site-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.15.1)\n",
      "Requirement already satisfied: joblib>=1.0.0 in /opt/homebrew/lib/python3.10/site-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/homebrew/lib/python3.10/site-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (3.1.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (22.1.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (6.0.2)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.8.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.3.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.2.0)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.3.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (3.19.0)\n",
      "Requirement already satisfied: marshmallow-enum<2.0.0,>=1.5.1 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (1.5.1)\n",
      "Requirement already satisfied: typing-inspect>=0.4.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (0.8.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets>=2.7.0->refuel-autolabel[openai]) (3.8.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /opt/homebrew/lib/python3.10/site-packages (from markdown-it-py<3.0.0,>=2.2.0->rich>=13.3.5->refuel-autolabel[openai]) (0.1.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=3.5.0->refuel-autolabel[openai]) (1.12.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/homebrew/lib/python3.10/site-packages (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (0.4.3)\n"
     ]
    }
   ],
   "source": [
    "!pip install 'refuel-autolabel[openai]'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c5b1630",
   "metadata": {},
   "source": [
    "#### Download the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9382044e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading example dataset from https://autolabel-benchmarking.s3.us-west-2.amazonaws.com/ethos/seed.csv to seed.csv...\n",
      "Downloading example dataset from https://autolabel-benchmarking.s3.us-west-2.amazonaws.com/ethos/test.csv to test.csv...\n",
      "100% [........................................] [63485/63485] bytes\r"
     ]
    }
   ],
   "source": [
    "from autolabel import get_data\n",
    "\n",
    "get_data(\"ethos\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c72542ae",
   "metadata": {},
   "source": [
    "This downloads two datasets:\n",
    "* `test.csv`: This is the larger dataset we are trying to label using LLMs\n",
    "* `seed.csv`: This is a small dataset where we already have human-provided labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffb92a79",
   "metadata": {},
   "source": [
    "## Start the labeling process!\n",
    "\n",
    "Labeling with Autolabel is a 3-step process:\n",
    "* First, we specify a labeling configuration (see `config.json` below)\n",
    "* Next, we do a dry-run on our dataset using the LLM specified in `config.json` by running `agent.plan`\n",
    "* Finally, we run the labeling with `agent.run`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84b014d1-f45c-4479-9acc-0d20870b1786",
   "metadata": {},
   "source": [
    "### First labeling run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c093fe91-3508-4140-8bd6-217034e3cce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from autolabel import LabelingAgent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c93fae0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the config\n",
    "with open(\"config_ethos.json\") as f:\n",
    "     config = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91dfc1f9",
   "metadata": {},
   "source": [
    "Let's review the configuration file below. You'll notice the following useful keys:\n",
    "* `task_type`: `attribute_extraction` (since it's an attribute extraction task)\n",
    "* `model`: `{'provider': 'openai', 'name': 'gpt-3.5-turbo'}` (use a specific OpenAI model)\n",
    "* `prompt.task_guidelines`: `'You are an expert at classifying hate speech and identifying...` (how we describe the task to the LLM)\n",
    "* `prompt.attributes`: `[\n",
    "            {\n",
    "                \"name\": \"violence\",\n",
    "                ...\n",
    "            },\n",
    "            {\n",
    "                \"name\": \"directed_vs_generalized\",\n",
    "                ...\n",
    "            },\n",
    "            {\n",
    "                \"name\": \"gender\",\n",
    "                ...\n",
    "            }\n",
    "        ]` (the full list of labels to choose from)\n",
    "* `prompt.few_shot_num`: 5 (how many labeled examples to provide to the LLM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "450ad645",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'task_name': 'EthosAttributeExtraction',\n",
       " 'task_type': 'attribute_extraction',\n",
       " 'dataset': {'text_column': 'text', 'delimiter': ','},\n",
       " 'model': {'provider': 'openai', 'name': 'gpt-3.5-turbo'},\n",
       " 'prompt': {'task_guidelines': 'You are an expert at classifying hate speech and identifying the type of hate speech. Read the following tweets and extract the following attributes from the text.',\n",
       "  'attributes': [{'name': 'violence',\n",
       "    'options': ['not_violent', 'violent'],\n",
       "    'description': 'If the tweet mentions violence towards a person or a group.'},\n",
       "   {'name': 'directed_vs_generalized',\n",
       "    'options': ['generalized', 'directed'],\n",
       "    'description': 'If the hate speech is generalized towards a group or directed towards a specific person.'},\n",
       "   {'name': 'gender',\n",
       "    'options': ['true', 'false'],\n",
       "    'description': 'If the hate speech uses gendered language and attacks a particular gender.'}],\n",
       "  'few_shot_examples': 'seed.csv',\n",
       "  'few_shot_selection': 'fixed',\n",
       "  'few_shot_num': 5,\n",
       "  'example_template': 'Text: {text}\\nOutput: {output_dict}'}}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "acb4a3de-fa84-4b94-b17a-7a6fac892a1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create an agent for labeling\n",
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "92667a39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "033eb2f9cc054ebf846314479c55fccb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $1.1186 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 428     </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0026 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$1.1186\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m428    \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0026\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">You are an expert at classifying hate speech and identifying the type of hate speech. Read the following tweets and\n",
       "extract the following attributes from the text.\n",
       "\n",
       "You will return the extracted attributes as a json with the following keys:\n",
       "<span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"If the tweet mentions violence towards a person or a group.\\nOptions:\\nnot_violent,violent\"</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"If the hate speech is generalized towards a group or directed towards a specific </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">person.\\nOptions:\\ngeneralized,directed\"</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"If the hate speech uses gendered language and attacks a particular gender.\\nOptions:\\ntrue,false\"</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Some examples with their output answers are provided below:\n",
       "\n",
       "Text: You should know women's sports are a joke\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"not_violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"generalized\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"true\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Text: You look like Sloth with deeper Down’s syndrome\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"not_violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"directed\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"false\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Text: You look like Russian and speak like Indian. Both are disgusting go kill yourself\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"directed\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"false\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Text: Women deserve to be abused, I guess.\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"generalized\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"true\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Text: Women are made for making babies and cooking dinner and nothing else!!!\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"not_violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"generalized\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"true\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Now I want you to label the following example:\n",
       "Text: why haven't you faggots taken the bi-pill yet\n",
       "Output: \n",
       "</pre>\n"
      ],
      "text/plain": [
       "You are an expert at classifying hate speech and identifying the type of hate speech. Read the following tweets and\n",
       "extract the following attributes from the text.\n",
       "\n",
       "You will return the extracted attributes as a json with the following keys:\n",
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m\"violence\"\u001b[0m: \u001b[32m\"If the tweet mentions violence towards a person or a group.\\nOptions:\\nnot_violent,violent\"\u001b[0m,\n",
       "    \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"If the hate speech is generalized towards a group or directed towards a specific \u001b[0m\n",
       "\u001b[32mperson.\\nOptions:\\ngeneralized,directed\"\u001b[0m,\n",
       "    \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"If the hate speech uses gendered language and attacks a particular gender.\\nOptions:\\ntrue,false\"\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n",
       "\n",
       "Some examples with their output answers are provided below:\n",
       "\n",
       "Text: You should know women's sports are a joke\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"not_violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"generalized\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"true\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Text: You look like Sloth with deeper Down’s syndrome\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"not_violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"directed\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"false\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Text: You look like Russian and speak like Indian. Both are disgusting go kill yourself\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"directed\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"false\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Text: Women deserve to be abused, I guess.\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"generalized\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"true\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Text: Women are made for making babies and cooking dinner and nothing else!!!\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"not_violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"generalized\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"true\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Now I want you to label the following example:\n",
       "Text: why haven't you faggots taken the bi-pill yet\n",
       "Output: \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dry-run -- this tells us how much this will cost and shows an example prompt\n",
    "from autolabel import AutolabelDataset\n",
    "\n",
    "ds = AutolabelDataset(\"data/ethos/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dd703025-54d8-4349-b0d6-736d2380e966",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "522f61d4d6f44496a37e35faf476850d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Actual Cost: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0665</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Actual Cost: \u001b[1;36m0.0665\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> violence:… </span>┃<span style=\"font-weight: bold\"> violence:… </span>┃<span style=\"font-weight: bold\"> violence:… </span>┃<span style=\"font-weight: bold\"> directed_… </span>┃<span style=\"font-weight: bold\"> directed… </span>┃<span style=\"font-weight: bold\"> directed_… </span>┃<span style=\"font-weight: bold\"> gender:s… </span>┃<span style=\"font-weight: bold\"> gender:co… </span>┃<span style=\"font-weight: bold\"> gender:a… </span>┃\n",
       "┡━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100        </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0        </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.89       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100        </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.89       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0        </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.94      </span>│\n",
       "└────────────┴────────────┴────────────┴────────────┴───────────┴────────────┴───────────┴────────────┴───────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mviolence:…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mviolence:…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mviolence:…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdirected_…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdirected…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdirected_…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mgender:s…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mgender:co…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mgender:a…\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m100       \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0       \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.89      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100       \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.89      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0       \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.94     \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└────────────┴────────────┴────────────┴────────────┴───────────┴────────────┴───────────┴────────────┴───────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# now, do the actual labeling\n",
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40e1901d",
   "metadata": {},
   "source": [
    "We are at 89% accuracy when labeling the first 100 examples for labeling violence and gender, and 94% for gender. Let's see if we can use confidence scores to improve accuracy further by removing the less confident examples from our labeled set."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d7645ab",
   "metadata": {},
   "source": [
    "### Compute confidence scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "30f2fb7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start computing confidence scores (using Refuel's LLMs)\n",
    "os.environ[\"REFUEL_API_KEY\"] = \"xxxxxxxxxxxxxxxxx\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5fbc1264",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set `compute_confidence` -> True\n",
    "config[\"model\"][\"compute_confidence\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1998f5e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "119e6f22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eccf18d75b174eb6ad95d04975d92f27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $1.1186 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 428     </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0026 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$1.1186\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m428    \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0026\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">You are an expert at classifying hate speech and identifying the type of hate speech. Read the following tweets and\n",
       "extract the following attributes from the text.\n",
       "\n",
       "You will return the extracted attributes as a json with the following keys:\n",
       "<span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"If the tweet mentions violence towards a person or a group.\\nOptions:\\nnot_violent,violent\"</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"If the hate speech is generalized towards a group or directed towards a specific </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">person.\\nOptions:\\ngeneralized,directed\"</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"If the hate speech uses gendered language and attacks a particular gender.\\nOptions:\\ntrue,false\"</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Some examples with their output answers are provided below:\n",
       "\n",
       "Text: You should know women's sports are a joke\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"not_violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"generalized\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"true\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Text: You look like Sloth with deeper Down’s syndrome\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"not_violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"directed\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"false\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Text: You look like Russian and speak like Indian. Both are disgusting go kill yourself\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"directed\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"false\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Text: Women deserve to be abused, I guess.\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"generalized\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"true\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Text: Women are made for making babies and cooking dinner and nothing else!!!\n",
       "Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"violence\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"not_violent\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"directed_vs_generalized\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"generalized\"</span>, <span style=\"color: #008000; text-decoration-color: #008000\">\"gender\"</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"true\"</span><span style=\"font-weight: bold\">}</span>\n",
       "\n",
       "Now I want you to label the following example:\n",
       "Text: why haven't you faggots taken the bi-pill yet\n",
       "Output: \n",
       "</pre>\n"
      ],
      "text/plain": [
       "You are an expert at classifying hate speech and identifying the type of hate speech. Read the following tweets and\n",
       "extract the following attributes from the text.\n",
       "\n",
       "You will return the extracted attributes as a json with the following keys:\n",
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m\"violence\"\u001b[0m: \u001b[32m\"If the tweet mentions violence towards a person or a group.\\nOptions:\\nnot_violent,violent\"\u001b[0m,\n",
       "    \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"If the hate speech is generalized towards a group or directed towards a specific \u001b[0m\n",
       "\u001b[32mperson.\\nOptions:\\ngeneralized,directed\"\u001b[0m,\n",
       "    \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"If the hate speech uses gendered language and attacks a particular gender.\\nOptions:\\ntrue,false\"\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n",
       "\n",
       "Some examples with their output answers are provided below:\n",
       "\n",
       "Text: You should know women's sports are a joke\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"not_violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"generalized\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"true\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Text: You look like Sloth with deeper Down’s syndrome\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"not_violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"directed\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"false\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Text: You look like Russian and speak like Indian. Both are disgusting go kill yourself\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"directed\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"false\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Text: Women deserve to be abused, I guess.\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"generalized\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"true\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Text: Women are made for making babies and cooking dinner and nothing else!!!\n",
       "Output: \u001b[1m{\u001b[0m\u001b[32m\"violence\"\u001b[0m: \u001b[32m\"not_violent\"\u001b[0m, \u001b[32m\"directed_vs_generalized\"\u001b[0m: \u001b[32m\"generalized\"\u001b[0m, \u001b[32m\"gender\"\u001b[0m: \u001b[32m\"true\"\u001b[0m\u001b[1m}\u001b[0m\n",
       "\n",
       "Now I want you to label the following example:\n",
       "Text: why haven't you faggots taken the bi-pill yet\n",
       "Output: \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from autolabel import AutolabelDataset\n",
    "\n",
    "ds = AutolabelDataset(\"data/ethos/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "254d179b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db880f82957b449d993ec1e36192d99c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Actual Cost: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Actual Cost: \u001b[1;36m0.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> viole… </span>┃<span style=\"font-weight: bold\"> violen… </span>┃<span style=\"font-weight: bold\"> viole… </span>┃<span style=\"font-weight: bold\"> violen… </span>┃<span style=\"font-weight: bold\"> direc… </span>┃<span style=\"font-weight: bold\"> direct… </span>┃<span style=\"font-weight: bold\"> direc… </span>┃<span style=\"font-weight: bold\"> direct… </span>┃<span style=\"font-weight: bold\"> gende… </span>┃<span style=\"font-weight: bold\"> gender… </span>┃<span style=\"font-weight: bold\"> gende… </span>┃<span style=\"font-weight: bold\"> gender… </span>┃\n",
       "┡━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.89   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7171  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.89   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6885  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.94   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8652  </span>│\n",
       "└────────┴─────────┴────────┴─────────┴────────┴─────────┴────────┴─────────┴────────┴─────────┴────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mviole…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mviolen…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mviole…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mviolen…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdirec…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdirect…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdirec…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdirect…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mgende…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mgender…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mgende…\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mgender…\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m100   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.89  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7171 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.89  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6885 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.94  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8652 \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└────────┴─────────┴────────┴─────────┴────────┴─────────┴────────┴─────────┴────────┴─────────┴────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5d4b597b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>text</th>\n",
       "      <th>violence</th>\n",
       "      <th>directed_vs_generalized</th>\n",
       "      <th>gender</th>\n",
       "      <th>EthosAttributeExtraction_label</th>\n",
       "      <th>violence_label</th>\n",
       "      <th>directed_vs_generalized_label</th>\n",
       "      <th>gender_label</th>\n",
       "      <th>EthosAttributeExtraction_error</th>\n",
       "      <th>EthosAttributeExtraction_prompt</th>\n",
       "      <th>EthosAttributeExtraction_successfully_labeled</th>\n",
       "      <th>EthosAttributeExtraction_annotation</th>\n",
       "      <th>EthosAttributeExtraction_confidence</th>\n",
       "      <th>violence_confidence</th>\n",
       "      <th>directed_vs_generalized_confidence</th>\n",
       "      <th>gender_confidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>why haven't you faggots taken the bi-pill yet</td>\n",
       "      <td>violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>{'violence': 'not_violent', 'directed_vs_gener...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.7650375365, 'directed_vs_genera...</td>\n",
       "      <td>0.765038</td>\n",
       "      <td>0.603027</td>\n",
       "      <td>0.836589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>Why don’t you go beck your god to murder you? ...</td>\n",
       "      <td>violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>{'violence': 'not_violent', 'directed_vs_gener...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.75377655025, 'directed_vs_gener...</td>\n",
       "      <td>0.753777</td>\n",
       "      <td>0.584473</td>\n",
       "      <td>0.925618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>why does that african american monkey hold gun...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>{'violence': 'not_violent', 'directed_vs_gener...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.79421997, 'directed_vs_generali...</td>\n",
       "      <td>0.794220</td>\n",
       "      <td>0.399414</td>\n",
       "      <td>0.898600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>Why does Bulgaria exist as a country? It's one...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>{'violence': 'not_violent', 'directed_vs_gener...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>generalized</td>\n",
       "      <td>false</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.825805665, 'directed_vs_general...</td>\n",
       "      <td>0.825806</td>\n",
       "      <td>0.873291</td>\n",
       "      <td>0.863770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>Why do negroes tear down their own neighborhoo...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>generalized</td>\n",
       "      <td>false</td>\n",
       "      <td>{'violence': 'not_violent', 'directed_vs_gener...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>generalized</td>\n",
       "      <td>false</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.7605590775, 'directed_vs_genera...</td>\n",
       "      <td>0.760559</td>\n",
       "      <td>0.715820</td>\n",
       "      <td>0.897949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>100</td>\n",
       "      <td>If I had Down syndrome I would kill myself</td>\n",
       "      <td>violent</td>\n",
       "      <td>generalized</td>\n",
       "      <td>false</td>\n",
       "      <td>{'violence': 'not_violent', 'directed_vs_gener...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>generalized</td>\n",
       "      <td>false</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.7584991455, 'directed_vs_genera...</td>\n",
       "      <td>0.758499</td>\n",
       "      <td>0.510834</td>\n",
       "      <td>0.953288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>101</td>\n",
       "      <td>I’m searching for a hitman right now to kill t...</td>\n",
       "      <td>violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>true</td>\n",
       "      <td>{'violence': 'violent', 'directed_vs_generaliz...</td>\n",
       "      <td>violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>true</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.9345703000000001, 'directed_vs_...</td>\n",
       "      <td>0.934570</td>\n",
       "      <td>0.791016</td>\n",
       "      <td>0.725342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>102</td>\n",
       "      <td>I remember once I was in charge of a disabled ...</td>\n",
       "      <td>violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>{'violence': 'violent', 'directed_vs_generaliz...</td>\n",
       "      <td>violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.9240722699999999, 'directed_vs_...</td>\n",
       "      <td>0.924072</td>\n",
       "      <td>0.851562</td>\n",
       "      <td>0.916016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>103</td>\n",
       "      <td>I just don't like Homosexuals raising kids it'...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>generalized</td>\n",
       "      <td>false</td>\n",
       "      <td>{'violence': 'not_violent', 'directed_vs_gener...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>generalized</td>\n",
       "      <td>false</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.821228025, 'directed_vs_general...</td>\n",
       "      <td>0.821228</td>\n",
       "      <td>0.780518</td>\n",
       "      <td>0.837240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>104</td>\n",
       "      <td>I hate mentally disabled people trying to touc...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>generalized</td>\n",
       "      <td>false</td>\n",
       "      <td>{'violence': 'not_violent', 'directed_vs_gener...</td>\n",
       "      <td>not_violent</td>\n",
       "      <td>directed</td>\n",
       "      <td>false</td>\n",
       "      <td>None</td>\n",
       "      <td>You are an expert at classifying hate speech a...</td>\n",
       "      <td>True</td>\n",
       "      <td>successfully_labeled=True label={'violence': '...</td>\n",
       "      <td>{'violence': 0.7784423825, 'directed_vs_genera...</td>\n",
       "      <td>0.778442</td>\n",
       "      <td>0.589355</td>\n",
       "      <td>0.873698</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                               text     violence  \\\n",
       "0           5      why haven't you faggots taken the bi-pill yet      violent   \n",
       "1           6  Why don’t you go beck your god to murder you? ...      violent   \n",
       "2           7  why does that african american monkey hold gun...  not_violent   \n",
       "3           8  Why does Bulgaria exist as a country? It's one...  not_violent   \n",
       "4           9  Why do negroes tear down their own neighborhoo...  not_violent   \n",
       "..        ...                                                ...          ...   \n",
       "95        100         If I had Down syndrome I would kill myself      violent   \n",
       "96        101  I’m searching for a hitman right now to kill t...      violent   \n",
       "97        102  I remember once I was in charge of a disabled ...      violent   \n",
       "98        103  I just don't like Homosexuals raising kids it'...  not_violent   \n",
       "99        104  I hate mentally disabled people trying to touc...  not_violent   \n",
       "\n",
       "   directed_vs_generalized gender  \\\n",
       "0                 directed  false   \n",
       "1                 directed  false   \n",
       "2                 directed  false   \n",
       "3                 directed  false   \n",
       "4              generalized  false   \n",
       "..                     ...    ...   \n",
       "95             generalized  false   \n",
       "96                directed   true   \n",
       "97                directed  false   \n",
       "98             generalized  false   \n",
       "99             generalized  false   \n",
       "\n",
       "                       EthosAttributeExtraction_label violence_label  \\\n",
       "0   {'violence': 'not_violent', 'directed_vs_gener...    not_violent   \n",
       "1   {'violence': 'not_violent', 'directed_vs_gener...    not_violent   \n",
       "2   {'violence': 'not_violent', 'directed_vs_gener...    not_violent   \n",
       "3   {'violence': 'not_violent', 'directed_vs_gener...    not_violent   \n",
       "4   {'violence': 'not_violent', 'directed_vs_gener...    not_violent   \n",
       "..                                                ...            ...   \n",
       "95  {'violence': 'not_violent', 'directed_vs_gener...    not_violent   \n",
       "96  {'violence': 'violent', 'directed_vs_generaliz...        violent   \n",
       "97  {'violence': 'violent', 'directed_vs_generaliz...        violent   \n",
       "98  {'violence': 'not_violent', 'directed_vs_gener...    not_violent   \n",
       "99  {'violence': 'not_violent', 'directed_vs_gener...    not_violent   \n",
       "\n",
       "   directed_vs_generalized_label gender_label EthosAttributeExtraction_error  \\\n",
       "0                       directed        false                           None   \n",
       "1                       directed        false                           None   \n",
       "2                       directed        false                           None   \n",
       "3                    generalized        false                           None   \n",
       "4                    generalized        false                           None   \n",
       "..                           ...          ...                            ...   \n",
       "95                   generalized        false                           None   \n",
       "96                      directed         true                           None   \n",
       "97                      directed        false                           None   \n",
       "98                   generalized        false                           None   \n",
       "99                      directed        false                           None   \n",
       "\n",
       "                      EthosAttributeExtraction_prompt  \\\n",
       "0   You are an expert at classifying hate speech a...   \n",
       "1   You are an expert at classifying hate speech a...   \n",
       "2   You are an expert at classifying hate speech a...   \n",
       "3   You are an expert at classifying hate speech a...   \n",
       "4   You are an expert at classifying hate speech a...   \n",
       "..                                                ...   \n",
       "95  You are an expert at classifying hate speech a...   \n",
       "96  You are an expert at classifying hate speech a...   \n",
       "97  You are an expert at classifying hate speech a...   \n",
       "98  You are an expert at classifying hate speech a...   \n",
       "99  You are an expert at classifying hate speech a...   \n",
       "\n",
       "    EthosAttributeExtraction_successfully_labeled  \\\n",
       "0                                            True   \n",
       "1                                            True   \n",
       "2                                            True   \n",
       "3                                            True   \n",
       "4                                            True   \n",
       "..                                            ...   \n",
       "95                                           True   \n",
       "96                                           True   \n",
       "97                                           True   \n",
       "98                                           True   \n",
       "99                                           True   \n",
       "\n",
       "                  EthosAttributeExtraction_annotation  \\\n",
       "0   successfully_labeled=True label={'violence': '...   \n",
       "1   successfully_labeled=True label={'violence': '...   \n",
       "2   successfully_labeled=True label={'violence': '...   \n",
       "3   successfully_labeled=True label={'violence': '...   \n",
       "4   successfully_labeled=True label={'violence': '...   \n",
       "..                                                ...   \n",
       "95  successfully_labeled=True label={'violence': '...   \n",
       "96  successfully_labeled=True label={'violence': '...   \n",
       "97  successfully_labeled=True label={'violence': '...   \n",
       "98  successfully_labeled=True label={'violence': '...   \n",
       "99  successfully_labeled=True label={'violence': '...   \n",
       "\n",
       "                  EthosAttributeExtraction_confidence  violence_confidence  \\\n",
       "0   {'violence': 0.7650375365, 'directed_vs_genera...             0.765038   \n",
       "1   {'violence': 0.75377655025, 'directed_vs_gener...             0.753777   \n",
       "2   {'violence': 0.79421997, 'directed_vs_generali...             0.794220   \n",
       "3   {'violence': 0.825805665, 'directed_vs_general...             0.825806   \n",
       "4   {'violence': 0.7605590775, 'directed_vs_genera...             0.760559   \n",
       "..                                                ...                  ...   \n",
       "95  {'violence': 0.7584991455, 'directed_vs_genera...             0.758499   \n",
       "96  {'violence': 0.9345703000000001, 'directed_vs_...             0.934570   \n",
       "97  {'violence': 0.9240722699999999, 'directed_vs_...             0.924072   \n",
       "98  {'violence': 0.821228025, 'directed_vs_general...             0.821228   \n",
       "99  {'violence': 0.7784423825, 'directed_vs_genera...             0.778442   \n",
       "\n",
       "    directed_vs_generalized_confidence  gender_confidence  \n",
       "0                             0.603027           0.836589  \n",
       "1                             0.584473           0.925618  \n",
       "2                             0.399414           0.898600  \n",
       "3                             0.873291           0.863770  \n",
       "4                             0.715820           0.897949  \n",
       "..                                 ...                ...  \n",
       "95                            0.510834           0.953288  \n",
       "96                            0.791016           0.725342  \n",
       "97                            0.851562           0.916016  \n",
       "98                            0.780518           0.837240  \n",
       "99                            0.589355           0.873698  \n",
       "\n",
       "[100 rows x 17 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds.df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a708932",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
